#!/bin/bash

deepspeed --num_gpus 4 ../../src/train_bash.py \
  --deepspeed ../deepspeed/ds_z3_config.json \
  --stage sft \
  --do_train \
  --model_name_or_path /home/image_team/image_team_docker_home/lgd/e_commerce_llm/weights/Qwen1.5-7B/ \
  --dataset alpaca_gpt4_data_zh_retained,sft_gpt4_livestreaming_data_20240105,sft_product_extraction_train,sn_generate_gpt_long_title_part0,sn_generate_gpt_long_title_part1,sn_generate_gpt_long_title_part2,sn_generate_gpt_seo_cp,sn_generate_gpt_seo_other,sn_generate_gpt_seo_phb,sn_generate_gpt_seo_zc,sn_generate_gpt_short_title_part0,sn_generate_gpt_short_title_part1,sn_generate_gpt_train_gds_title,sn_generate_gpt_train_multigds_xhs,sn_generate_gpt_train_part0,sn_generate_gpt_train_part1,sn_generate_qwen72b_gds_tj,who_are_you,tiktok_v1 \
  --dataset_dir ../../data \
  --template qwen \
  --finetuning_type lora \
  --lora_target q_proj,v_proj \
  --output_dir ../../saves/Qwen1.5-7B/lora/sft \
  --overwrite_cache \
  --overwrite_output_dir \
  --cutoff_len 1024 \
  --preprocessing_num_workers 16 \
  --per_device_train_batch_size 1 \
  --per_device_eval_batch_size 1 \
  --gradient_accumulation_steps 2 \
  --lr_scheduler_type cosine \
  --logging_steps 10 \
  --warmup_steps 20 \
  --save_steps 1000 \
  --eval_steps 1000 \
  --evaluation_strategy steps \
  --learning_rate 5e-5 \
  --num_train_epochs 3.0 \
  --max_samples 3000 \
  --val_size 0.1 \
  --ddp_timeout 180000000 \
  --plot_loss
